package com.yang.spark

import org.apache.spark.{SparkConf, SparkContext}

/**
  * Created by yang on 2018/6/11.
  * spark实现单词计数
  */
object wordcount {

  def main(args: Array[String]): Unit = {

    val conf = new SparkConf().setAppName("wordcount")
    val sc = new SparkContext(conf)
    val a = new SparkContext(conf)

    val path = if(args(0).endsWith("/")) args(0) else args(0) + "/"
    val data = sc.textFile(path).flatMap(_.split(" ")).filter(x => x != "").map(p => (p,1))
    val result = data.reduceByKey(_ + _)
    result.repartition(1).saveAsTextFile(args(1),classOf[org.apache.hadoop.io.compress.GzipCodec])

    sc.stop()
  }

}
